{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 144,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.metrics import f1_score\n",
    "\n",
    "train1 = np.loadtxt('data/model_save/train_bert_prob_0.7111825771920298.txt')\n",
    "train3 = np.loadtxt('data/model_save/train_bert_prob_0.7066069981686462.txt')\n",
    "train4 = np.loadtxt('data/model_save/train_bert_prob_0.7109690769378036.txt')\n",
    "test1 = np.loadtxt('data/model_save/test_bert_prob_0.7111825771920298.txt')\n",
    "test3 = np.loadtxt('data/model_save/test_bert_prob_0.7066069981686462.txt')\n",
    "test4 = np.loadtxt('data/model_save/test_bert_prob_0.7109690769378036.txt')\n",
    "\n",
    "train = pd.read_csv('data/nCoV_100k_train.labled.csv', header=0)\n",
    "test = pd.read_csv('data/nCov_10k_test.csv', header=0)\n",
    "train = train[(train['情感倾向'].isin(['-1', '0', '1']))]\n",
    "train = train[~train.index.isin(['35202', '41624', '33936'])].reset_index(drop=True)\n",
    "test = pd.read_csv('data/nCov_10k_test.csv',header=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 145,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "oof_train = 0.4*train1+0.3*train3+0.3*train4\n",
    "oof_test = 0.4*test1+0.3*test3+0.3*test4\n",
    "train['y'] = np.argmax(oof_train, axis=1) - 1\n",
    "test['y'] = np.argmax(oof_test, axis=1) - 1"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 146,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7258855894504425\n"
     ]
    }
   ],
   "source": [
    "train_y = train['情感倾向'].astype('int')+1\n",
    "\n",
    "cv_score = f1_score(train_y, np.argmax(oof_train, axis=-1), average='macro')\n",
    "print(cv_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 147,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_best = np.loadtxt('data/model_save2/test_bert_prob_0.7354800953858035.txt')\n",
    "test['y3'] = np.argmax(test_best, axis=1) - 1 "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 148,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0.9692"
      ]
     },
     "execution_count": 148,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test[test['y'] == test['y3']].shape[0]/test.shape[0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# list(test[test['y'] != test['y2']]['微博中文内容'].head(50))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 149,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# test[test['y'] != test['y2']][['微博中文内容','y','y2']]\n",
    "np.savetxt('./data/model_save/train_bert_prob_{}.txt'.format(cv_score), oof_train)\n",
    "np.savetxt('./data/model_save/test_bert_prob_{}.txt'.format(cv_score), oof_test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 150,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test['id'] = test['微博id'].copy()\n",
    "test[['id', 'y']].to_csv('./data/submit/bert_{}.csv'.format(cv_score), index=False)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 153,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7257441309518783\n"
     ]
    }
   ],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.metrics import f1_score\n",
    "\n",
    "train1 = np.loadtxt('data/model_save/train_bert_prob_0.7328015743130485.txt')\n",
    "train3 = np.loadtxt('data/model_save/train_bert_prob_0.7026235410914827.txt')\n",
    "train4 = np.loadtxt('data/model_save/train_bert_prob_0.7062602082447832.txt')\n",
    "test1 = np.loadtxt('data/model_save/test_bert_prob_0.7328015743130485.txt')\n",
    "test3 = np.loadtxt('data/model_save/test_bert_prob_0.7026235410914827.txt')\n",
    "test4 = np.loadtxt('data/model_save/test_bert_prob_0.7062602082447832.txt')\n",
    "\n",
    "train = pd.read_csv('data/nCoV_100k_train.labled.csv', header=0)\n",
    "test = pd.read_csv('data/nCov_10k_test.csv', header=0)\n",
    "train = train[(train['情感倾向'].isin(['-1', '0', '1']))]\n",
    "train = train[~train.index.isin(['35202', '41624', '33936'])].reset_index(drop=True)\n",
    "test = pd.read_csv('data/nCov_10k_test.csv',header=0)\n",
    "\n",
    "oof_train = 0.2*train1+0.4*train3+0.4*train4\n",
    "oof_test = 0.2*test1+0.4*test3+0.4*test4\n",
    "train['y'] = np.argmax(oof_train, axis=1) - 1\n",
    "test['y'] = np.argmax(oof_test, axis=1) - 1\n",
    "\n",
    "train_y = train['情感倾向'].astype('int')+1\n",
    "\n",
    "cv_score = f1_score(train_y, np.argmax(oof_train, axis=-1), average='macro')\n",
    "print(cv_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 24,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "0.7218571449930148\n"
     ]
    }
   ],
   "source": [
    "## 二次融合：投票\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.metrics import f1_score\n",
    "\n",
    "train1 = np.loadtxt('data/model_save/train_bert_prob_0.7058588365950068.txt')\n",
    "train3 = np.loadtxt('data/model_save2/train_bert_prob_0.7354800953858035.txt')\n",
    "test1 = np.loadtxt('data/model_save/test_bert_prob_0.7058588365950068.txt')\n",
    "test3 = np.loadtxt('data/model_save2/test_bert_prob_0.7354800953858035.txt')\n",
    "\n",
    "train = pd.read_csv('data/nCoV_100k_train.labled.csv', header=0)\n",
    "test = pd.read_csv('data/nCov_10k_test.csv', header=0)\n",
    "train = train[(train['情感倾向'].isin(['-1', '0', '1']))]\n",
    "train = train[~train.index.isin(['35202', '41624', '33936'])].reset_index(drop=True)\n",
    "test = pd.read_csv('data/nCov_10k_test.csv',header=0)\n",
    "\n",
    "oof_train = 0.65*train1+0.35*train3\n",
    "oof_test = 0.65*test1+0.35*test3\n",
    "train['y'] = np.argmax(oof_train, axis=1) - 1\n",
    "test['y'] = np.argmax(oof_test, axis=1) - 1\n",
    "\n",
    "train_y = train['情感倾向'].astype('int')+1\n",
    "\n",
    "cv_score = f1_score(train_y, np.argmax(oof_train, axis=-1), average='macro')\n",
    "print(cv_score)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 25,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       " 0    0.5778\n",
       " 1    0.2463\n",
       "-1    0.1759\n",
       "Name: y, dtype: float64"
      ]
     },
     "execution_count": 25,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "test['y'].value_counts(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 26,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test['id'] = test['微博id'].copy()\n",
    "test[['id', 'y']].to_csv('./data/submit/bert_{}.csv'.format(cv_score), index=False)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "## 0.7424"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 163,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import os\n",
    "import path\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from collections import Counter\n",
    "\n",
    "sub_path = './data/submit/'\n",
    "\n",
    "sub1 = pd.read_csv(sub_path + 'bert_0.7354800953858035.csv', encoding='utf-8')\n",
    "sub2 = pd.read_csv(sub_path + 'bert_0.7058588365950068.csv', encoding='utf-8')\n",
    "sub3 = pd.read_csv(sub_path + 'bert_0.7241026542291286.csv', encoding='utf-8')\n",
    "# sub4 = pd.read_csv(sub_path + 'bert_0.7241026542291286.csv', encoding='utf-8')\n",
    "# sub5 = pd.read_csv(sub_path + 'bert_0.7111825771920298.csv', encoding='utf-8')\n",
    "\n",
    "sub1.columns = ['id', 'y1']\n",
    "sub2.columns = ['id', 'y2'] \n",
    "sub3.columns = ['id', 'y3']\n",
    "\n",
    "sub = sub1.merge(sub2, on='id', how='left')\n",
    "sub = sub.merge(sub3, on='id', how='left')\n",
    "\n",
    "def vote1(value_1, value_2, value_3):\n",
    "    count = Counter()\n",
    "    count[value_1] += 1\n",
    "    count[value_2] += 1\n",
    "    count[value_3] += 1\n",
    "    # print(count)\n",
    "    return count.most_common(1)[0][0]\n",
    "def vote(value_1, value_2, value_3,value_4,value_5):\n",
    "    count = Counter()\n",
    "    count[value_1] += 1\n",
    "    count[value_2] += 1\n",
    "    count[value_3] += 1\n",
    "    count[value_4] += 1\n",
    "    count[value_5] += 1\n",
    "    print(count)\n",
    "    return count.most_common(1)[0][0]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 166,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sub['y'] = sub.apply(lambda index: vote1(index.y1, index.y2, index.y3), axis=1)\n",
    "sub[['id', 'y']].to_csv('./data/submit/submit_vote2.csv', encoding='utf-8', index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 157,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       " 0    0.5820\n",
       " 1    0.2373\n",
       "-1    0.1807\n",
       "Name: y2, dtype: float64"
      ]
     },
     "execution_count": 157,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub2['y2'].value_counts(1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 165,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/html": [
       "<div>\n",
       "<style scoped>\n",
       "    .dataframe tbody tr th:only-of-type {\n",
       "        vertical-align: middle;\n",
       "    }\n",
       "\n",
       "    .dataframe tbody tr th {\n",
       "        vertical-align: top;\n",
       "    }\n",
       "\n",
       "    .dataframe thead th {\n",
       "        text-align: right;\n",
       "    }\n",
       "</style>\n",
       "<table border=\"1\" class=\"dataframe\">\n",
       "  <thead>\n",
       "    <tr style=\"text-align: right;\">\n",
       "      <th></th>\n",
       "      <th>id</th>\n",
       "      <th>y1</th>\n",
       "      <th>y2</th>\n",
       "      <th>y3</th>\n",
       "      <th>y</th>\n",
       "    </tr>\n",
       "  </thead>\n",
       "  <tbody>\n",
       "    <tr>\n",
       "      <th>0</th>\n",
       "      <td>4456068992182160</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>1</th>\n",
       "      <td>4456424178427250</td>\n",
       "      <td>0</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>2</th>\n",
       "      <td>4456797466940200</td>\n",
       "      <td>0</td>\n",
       "      <td>-1</td>\n",
       "      <td>0</td>\n",
       "      <td>0</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>3</th>\n",
       "      <td>4456791021108920</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "    <tr>\n",
       "      <th>4</th>\n",
       "      <td>4457086404997440</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "      <td>-1</td>\n",
       "    </tr>\n",
       "  </tbody>\n",
       "</table>\n",
       "</div>"
      ],
      "text/plain": [
       "                 id  y1  y2  y3  y\n",
       "0  4456068992182160   0   0   0  0\n",
       "1  4456424178427250   0  -1  -1 -1\n",
       "2  4456797466940200   0  -1   0  0\n",
       "3  4456791021108920  -1  -1  -1 -1\n",
       "4  4457086404997440  -1  -1  -1 -1"
      ]
     },
     "execution_count": 165,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub.head()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "### 0.7436"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "                    id  y1  y2  y3  y4  y5\n",
      "0     4456068992182160   0   0   0   0   0\n",
      "1     4456424178427250  -1  -1   0  -1  -1\n",
      "2     4456797466940200  -1   0   0   0   0\n",
      "3     4456791021108920  -1  -1  -1  -1  -1\n",
      "4     4457086404997440  -1  -1  -1  -1  -1\n",
      "...                ...  ..  ..  ..  ..  ..\n",
      "9995  4464179518243680   0   0   0   0   0\n",
      "9996  4464274073923100   0   0   0   0   0\n",
      "9997  4464289160945130   0   0   0   0   0\n",
      "9998  4465347950314820   0   0   0   0   0\n",
      "9999  4465492650005690   0   0   0   0   0\n",
      "\n",
      "[10000 rows x 6 columns]\n"
     ]
    }
   ],
   "source": [
    "import os\n",
    "import path\n",
    "import numpy as np\n",
    "import pandas as pd\n",
    "from collections import Counter\n",
    "\n",
    "sub_path = './data/submit/'\n",
    "\n",
    "sub1 = pd.read_csv(sub_path + 'bert_0.7058588365950068.csv', encoding='utf-8')\n",
    "sub2 = pd.read_csv(sub_path + 'bert_0.735319424187954.csv', encoding='utf-8')\n",
    "sub3 = pd.read_csv(sub_path + 'bert_0.7354800953858035.csv', encoding='utf-8')\n",
    "# sub4 = pd.read_csv(sub_path + 'bert_0.7241026542291286.csv', encoding='utf-8')\n",
    "# sub5 = pd.read_csv(sub_path + 'bert_0.7111825771920298.csv', encoding='utf-8')\n",
    "\n",
    "sub1.columns = ['id', 'y1']\n",
    "sub2.columns = ['id', 'y2'] \n",
    "sub3.columns = ['id', 'y3']\n",
    "sub4.columns = ['id', 'y4']\n",
    "sub5.columns = ['id', 'y5']\n",
    "\n",
    "sub = sub1.merge(sub2, on='id', how='left')\n",
    "sub = sub.merge(sub3, on='id', how='left')\n",
    "# sub = sub.merge(sub4, on='id', how='left')\n",
    "# sub = sub.merge(sub5, on='id', how='left')\n",
    "\n",
    "def vote1(value_1, value_2, value_3):\n",
    "    count = Counter()\n",
    "    count[value_1] += 1\n",
    "    count[value_2] += 1\n",
    "    count[value_3] += 1\n",
    "    # print(count)\n",
    "    return count.most_common(1)[0][0]\n",
    "def vote(value_1, value_2, value_3,value_4,value_5):\n",
    "    count = Counter()\n",
    "    count[value_1] += 1\n",
    "    count[value_2] += 1\n",
    "    count[value_3] += 1\n",
    "    count[value_4] += 1\n",
    "    count[value_5] += 1\n",
    "    # print(count)\n",
    "    return count.most_common(1)[0][0]\n",
    "sub['y'] = sub.apply(lambda index: vote1(index.y1, index.y2, index.y3), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 21,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sub['y'] = sub.apply(lambda index: vote(index.y1, index.y2, index.y3, index.y4, index.y5), axis=1)\n",
    "# sub['y'] = sub.apply(lambda index: vote1(index.y1, index.y2, index.y3), axis=1)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 160,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "sub[['id', 'y']].to_csv('./data/submit/submit_vote2.csv', encoding='utf-8', index=None)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 155,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       " 0    0.5820\n",
       " 1    0.2383\n",
       "-1    0.1797\n",
       "Name: y, dtype: float64"
      ]
     },
     "execution_count": 155,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "sub['y'].value_counts(1)"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {
    "collapsed": true
   },
   "source": [
    "## 修复"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 54,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import pandas as pd\n",
    "from sklearn.metrics import f1_score\n",
    "\n",
    "# label_train = pd.read_csv('data/nCoV_100k_train.labled.csv', header=0)\n",
    "# unlabel_train = pd.read_csv('data/nCoV_900k_train.unlabled.csv', header=0)\n",
    "# test = pd.read_csv('data/nCov_10k_test.csv', header=0)\n",
    "# col = ['微博id', '微博中文内容']\n",
    "# train = pd.concat([label_train[col], unlabel_train[col]])\n",
    "# train = pd.concat([train, test[col]])\n",
    "# train.drop_duplicates(subset=['微博中文内容'], inplace=True)\n",
    "\n",
    "train = pd.read_csv('data/nCoV_100k_train.labled.csv', header=0)\n",
    "train = train[(train['情感倾向'].isin(['-1', '0', '1']))]\n",
    "train['情感倾向'] = train['情感倾向'].astype(int)\n",
    "train = train.drop_duplicates(subset=['微博中文内容'])\n",
    "train = train[train.notnull()][['微博中文内容','情感倾向']]\n",
    "test_repair = test.merge(train,how='inner',on='微博中文内容')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 65,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "submit = pd.read_csv('data/submit/submit.csv')\n",
    "submit.rename(columns={'id': '微博id', 'y':'情感倾向'}, inplace=True)\n",
    "# test['微博id'] = test_repair['微博id'].astype(int)\n",
    "submit2 = submit.merge(test_repair[['微博id','情感倾向']], how='inner', on='微博id')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 86,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train = pd.read_csv('data/nCoV_100k_train.labled.csv', header=0)\n",
    "train = train[(train['情感倾向'].isin(['-1', '0', '1']))]\n",
    "train = train[~train.index.isin(['35202', '41624', '33936'])].reset_index(drop=True)\n",
    "\n",
    "train['情感倾向'] = train['情感倾向'].astype(int)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 139,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "submit_repair = submit2[submit2['情感倾向_x'] != submit2['情感倾向_y']]#['id']\n",
    "train2 = train.merge(submit_repair[['微博id', '情感倾向_x']],on='微博id',how='left')\n",
    "train2['情感倾向'] = list(map(lambda x,y: y if y in [-1,0,1] else x, train2['情感倾向'], train2['情感倾向_x']))\n",
    "train2.drop(columns=['情感倾向_x'], axis=1, inplace=True)\n",
    "\n",
    "# train2.to_csv('data/new_train.csv', index=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 104,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "test_drop_id = list(submit2['微博id'])\n",
    "test['微博id'] = test['微博id'].astype('int')\n",
    "submit3 = submit[~submit['微博id'].isin(test_drop_id)]\n",
    "submit4 = test.merge(submit3, on='微博id', how='inner')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 111,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "new_train_presto = pd.concat([train2, submit4])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 114,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "new_train_presto.to_csv('data/new_train_presto.csv', index=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 141,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "Index(['微博id', '微博发布时间', '发布人账号', '微博中文内容', '微博图片', '微博视频', '情感倾向'], dtype='object')"
      ]
     },
     "execution_count": 141,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train2.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 142,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# submit4.columns,submit4.columns"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 121,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# submit = pd.concat([submit, test[['id', 'y']]])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 122,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# submit.to_csv('data/submit/submit_repair.csv',index=0)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 118,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# submit[submit['y_x'] != submit['y_y']][:10]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 119,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "# list(test[test['id'] == 4468031901960900]['微博中文内容'])"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 123,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": [
    "train = pd.read_csv('data/nCoV_100k_train.labled.csv', header=0)\n",
    "test = pd.read_csv('data/nCov_10k_test.csv', header=0)\n",
    "train = train[(train['情感倾向'].isin(['-1', '0', '1']))]\n",
    "train = train[~train.index.isin(['35202', '41624', '33936'])].reset_index(drop=True)\n",
    "test = pd.read_csv('data/test_repair.csv')\n",
    "test.rename(columns={'微博id': 'id', '情感倾向':'y'}, inplace=True)\n",
    "test_drop_id = list(test[test['y'].notnull()]['id'])\n",
    "train = train[~train['微博id'].isin(test_drop_id)]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 125,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "0     57353\n",
       "1     25284\n",
       "-1    16826\n",
       "Name: 情感倾向, dtype: int64"
      ]
     },
     "execution_count": 125,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train['情感倾向'].value_counts()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "collapsed": true
   },
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
